package downloader.biquge

import org.slf4j.LoggerFactory
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.selector.Selectable

/**
 * Created by dm on 2017/5/19 0019.
 */



class BiqugePageProcessor : PageProcessor {

    val Log = LoggerFactory.getLogger(javaClass)
    val urls = mutableMapOf<String,Int>()
    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(1000).setCycleRetryTimes(3).setTimeOut(300_000).setRetrySleepTime(10_000)
    }

    override fun process(page: us.codecraft.webmagic.Page) {
        val html = page.html
        val mainInfo: Selectable = html.xpath("//div[@id='maininfo']")

        if (mainInfo.match()) {
            val all = html.xpath("//div[@id='list']").links().all()
            all.forEachIndexed{i,s -> urls[s] = i}
            page.addTargetRequests(all)

        } else {
            val title = html.xpath("//div[@class='bookname']/h1/text()").get()
            page.putField("title", title)
            var content = html.xpath("//div[@id='content']/text()").get().replace(Regex("\\s+"),"\r\n").replace("\u00a0","")
            page.putField("content", content)
            page.putField("idx",urls.getValue(page.url.get()))

        }


    }

}